In [43]:
gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Not connected to a GPU')
else:
  print(gpu_info)
Tue Apr  5 15:57:09 2022       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   36C    P0    34W / 250W |  15981MiB / 16280MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+
In [2]:
from google.colab import drive
drive.mount('/content/drive')
import os
os.chdir(r'/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp')  # Change the directory to torchRay-master folder
# %reset
Mounted at /content/drive

Project B: Knowledge Distillation for Building Lightweight Deep Learning Models in Visual Classification Tasks

In [3]:
!pip install keras-flops
from keras_flops import get_flops
Collecting keras-flops
  Downloading keras_flops-0.1.2-py3-none-any.whl (5.3 kB)
Requirement already satisfied: tensorflow<3.0,>=2.2 in /usr/local/lib/python3.7/dist-packages (from keras-flops) (2.8.0)
Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (3.10.0.2)
Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (3.3.0)
Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (3.17.3)
Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.21.5)
Requirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (2.8.0)
Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (0.24.0)
Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (3.1.0)
Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.1.2)
Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (0.5.3)
Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (2.8.0)
Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.6.3)
Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.44.0)
Collecting tf-estimator-nightly==2.8.0.dev2021122109
  Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)
     |████████████████████████████████| 462 kB 8.2 MB/s 
Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.15.0)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.1.0)
Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (2.0)
Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (57.4.0)
Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.0.0)
Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (0.2.0)
Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (13.0.0)
Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<3.0,>=2.2->keras-flops) (1.14.0)
Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<3.0,>=2.2->keras-flops) (0.37.1)
Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<3.0,>=2.2->keras-flops) (1.5.2)
Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (0.6.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (1.0.1)
Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (1.8.1)
Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (2.23.0)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (3.3.6)
Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (1.35.0)
Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (0.4.6)
Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (4.2.4)
Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (4.8)
Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (0.2.8)
Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (1.3.1)
Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (4.11.3)
Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (3.7.0)
Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (0.4.8)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (3.0.4)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (2.10)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (1.24.3)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (2021.10.8)
Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<3.0,>=2.2->keras-flops) (3.2.0)
Installing collected packages: tf-estimator-nightly, keras-flops
Successfully installed keras-flops-0.1.2 tf-estimator-nightly-2.8.0.dev2021122109
In [4]:
import tensorflow.compat.v2 as tf
import tensorflow_datasets as tfds
from typing import Union
from keras.models import load_model
import matplotlib.pyplot as plt
import tensorflow.keras.backend as kb
from tensorflow.keras.utils import plot_model

tf.enable_v2_behavior()
NUM_EPOCHS = 12
In [5]:
import tensorflow
import keras
from keras.models import Sequential
from keras.layers import Conv1D, MaxPooling1D, Dense, Flatten,Conv2D,MaxPooling2D,Dropout
import matplotlib.pyplot as plt

import numpy as np

%load_ext tensorboard
import datetime

Data loading

In [6]:
import csv
import pandas as pd

df = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp/mhist_dataset/annotations.csv')

from PIL import Image
# load the image
image = Image.open('/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp/mhist_dataset/images/'+'MHIST_aaa.png')
# summarize some details about the image
print(image.format)
print(image.mode)
print(image.size)
# show the image
image.show()

# load and display an image with Matplotlib
from matplotlib import image
from matplotlib import pyplot
# load image as pixel array
data = image.imread('/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp/mhist_dataset/images/'+'MHIST_aaa.png')
# summarize shape of the pixel array
print(data.dtype)
print(data.shape)
# display the array of pixels as an image
pyplot.imshow(data)
pyplot.show()

# load all images in a directory
from os import listdir
from matplotlib import image
# load all images in a directory
train_images_SSA = list()
train_images_HP = list()
test_images_SSA = list()
test_images_HP = list()

for filename in listdir('/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp/mhist_dataset/images'):
    img_data = image.imread('/content/drive/MyDrive/Colab Notebooks/1512/Project_B_Supp/mhist_dataset/images/' + filename)
    i=df[df['Image Name']==filename]
    if(i['Majority Vote Label'][i.index.values[0]] == 'SSA' and i['Partition'][i.index.values[0]] == 'train'):
        # print('This picture belong to SSA.')
        train_images_SSA.append(img_data)
    elif(i['Majority Vote Label'][i.index.values[0]] == 'SSA' and i['Partition'][i.index.values[0]] == 'test'):
        # print('This picture belong to HP.')
        test_images_SSA.append(img_data)
    elif(i['Majority Vote Label'][i.index.values[0]] == 'HP' and i['Partition'][i.index.values[0]] == 'train'):
        train_images_HP.append(img_data)
    elif(i['Majority Vote Label'][i.index.values[0]] == 'HP' and i['Partition'][i.index.values[0]] == 'test'):
        test_images_HP.append(img_data)


np_train_HP = np.array(train_images_HP)
np_train_SSA = np.array(train_images_SSA)
np_test_HP = np.array(test_images_HP)
np_test_SSA = np.array(test_images_SSA)
print(np_train_HP.shape)
print(np_train_SSA.shape)
print(np_test_HP.shape)
print(np_test_SSA.shape)

np_train_set = np.vstack((np_train_HP,np_train_SSA))
np_train_label = np.vstack((np.zeros([np_train_HP.shape[0],1]),np.ones([np_train_SSA.shape[0],1])))
np_train_label = tensorflow.keras.utils.to_categorical(np_train_label,2)
np_test_set = np.vstack((np_test_HP,np_test_SSA))
np_test_label = np.vstack((np.zeros([np_test_HP.shape[0],1]),np.ones([np_test_SSA.shape[0],1])))
np_test_label = tensorflow.keras.utils.to_categorical(np_test_label,2)
del np_train_HP
del np_train_SSA
del np_test_HP
del np_test_SSA

del train_images_SSA
del train_images_HP
del test_images_SSA
del test_images_HP
PNG
RGB
(224, 224)
float32
(224, 224, 3)
(1545, 224, 224, 3)
(630, 224, 224, 3)
(617, 224, 224, 3)
(360, 224, 224, 3)
In [29]:
train_idx = np.random.permutation(len(np_train_set))
np_train_set,np_train_label = np_train_set[train_idx], np_train_label[train_idx]
# print(np_train_label)

Model creation

In [44]:
res_model = tensorflow.keras.applications.ResNet50V2(include_top=False,weights="imagenet",input_shape=(224,224,3))
for layer in res_model.layers:
    layer.trainable = False

for i, layer in enumerate(res_model.layers):
    print(i, layer.name, "-", layer.trainable)

mobile_model = tensorflow.keras.applications.mobilenet_v2.MobileNetV2(include_top=False,weights="imagenet",input_shape=(224,224,3))
for layer in mobile_model.layers:
    layer.trainable = False

for i, layer in enumerate(mobile_model.layers):
    print(i, layer.name, "-", layer.trainable)
0 input_1 - False
1 conv1_pad - False
2 conv1_conv - False
3 pool1_pad - False
4 pool1_pool - False
5 conv2_block1_preact_bn - False
6 conv2_block1_preact_relu - False
7 conv2_block1_1_conv - False
8 conv2_block1_1_bn - False
9 conv2_block1_1_relu - False
10 conv2_block1_2_pad - False
11 conv2_block1_2_conv - False
12 conv2_block1_2_bn - False
13 conv2_block1_2_relu - False
14 conv2_block1_0_conv - False
15 conv2_block1_3_conv - False
16 conv2_block1_out - False
17 conv2_block2_preact_bn - False
18 conv2_block2_preact_relu - False
19 conv2_block2_1_conv - False
20 conv2_block2_1_bn - False
21 conv2_block2_1_relu - False
22 conv2_block2_2_pad - False
23 conv2_block2_2_conv - False
24 conv2_block2_2_bn - False
25 conv2_block2_2_relu - False
26 conv2_block2_3_conv - False
27 conv2_block2_out - False
28 conv2_block3_preact_bn - False
29 conv2_block3_preact_relu - False
30 conv2_block3_1_conv - False
31 conv2_block3_1_bn - False
32 conv2_block3_1_relu - False
33 conv2_block3_2_pad - False
34 conv2_block3_2_conv - False
35 conv2_block3_2_bn - False
36 conv2_block3_2_relu - False
37 max_pooling2d - False
38 conv2_block3_3_conv - False
39 conv2_block3_out - False
40 conv3_block1_preact_bn - False
41 conv3_block1_preact_relu - False
42 conv3_block1_1_conv - False
43 conv3_block1_1_bn - False
44 conv3_block1_1_relu - False
45 conv3_block1_2_pad - False
46 conv3_block1_2_conv - False
47 conv3_block1_2_bn - False
48 conv3_block1_2_relu - False
49 conv3_block1_0_conv - False
50 conv3_block1_3_conv - False
51 conv3_block1_out - False
52 conv3_block2_preact_bn - False
53 conv3_block2_preact_relu - False
54 conv3_block2_1_conv - False
55 conv3_block2_1_bn - False
56 conv3_block2_1_relu - False
57 conv3_block2_2_pad - False
58 conv3_block2_2_conv - False
59 conv3_block2_2_bn - False
60 conv3_block2_2_relu - False
61 conv3_block2_3_conv - False
62 conv3_block2_out - False
63 conv3_block3_preact_bn - False
64 conv3_block3_preact_relu - False
65 conv3_block3_1_conv - False
66 conv3_block3_1_bn - False
67 conv3_block3_1_relu - False
68 conv3_block3_2_pad - False
69 conv3_block3_2_conv - False
70 conv3_block3_2_bn - False
71 conv3_block3_2_relu - False
72 conv3_block3_3_conv - False
73 conv3_block3_out - False
74 conv3_block4_preact_bn - False
75 conv3_block4_preact_relu - False
76 conv3_block4_1_conv - False
77 conv3_block4_1_bn - False
78 conv3_block4_1_relu - False
79 conv3_block4_2_pad - False
80 conv3_block4_2_conv - False
81 conv3_block4_2_bn - False
82 conv3_block4_2_relu - False
83 max_pooling2d_1 - False
84 conv3_block4_3_conv - False
85 conv3_block4_out - False
86 conv4_block1_preact_bn - False
87 conv4_block1_preact_relu - False
88 conv4_block1_1_conv - False
89 conv4_block1_1_bn - False
90 conv4_block1_1_relu - False
91 conv4_block1_2_pad - False
92 conv4_block1_2_conv - False
93 conv4_block1_2_bn - False
94 conv4_block1_2_relu - False
95 conv4_block1_0_conv - False
96 conv4_block1_3_conv - False
97 conv4_block1_out - False
98 conv4_block2_preact_bn - False
99 conv4_block2_preact_relu - False
100 conv4_block2_1_conv - False
101 conv4_block2_1_bn - False
102 conv4_block2_1_relu - False
103 conv4_block2_2_pad - False
104 conv4_block2_2_conv - False
105 conv4_block2_2_bn - False
106 conv4_block2_2_relu - False
107 conv4_block2_3_conv - False
108 conv4_block2_out - False
109 conv4_block3_preact_bn - False
110 conv4_block3_preact_relu - False
111 conv4_block3_1_conv - False
112 conv4_block3_1_bn - False
113 conv4_block3_1_relu - False
114 conv4_block3_2_pad - False
115 conv4_block3_2_conv - False
116 conv4_block3_2_bn - False
117 conv4_block3_2_relu - False
118 conv4_block3_3_conv - False
119 conv4_block3_out - False
120 conv4_block4_preact_bn - False
121 conv4_block4_preact_relu - False
122 conv4_block4_1_conv - False
123 conv4_block4_1_bn - False
124 conv4_block4_1_relu - False
125 conv4_block4_2_pad - False
126 conv4_block4_2_conv - False
127 conv4_block4_2_bn - False
128 conv4_block4_2_relu - False
129 conv4_block4_3_conv - False
130 conv4_block4_out - False
131 conv4_block5_preact_bn - False
132 conv4_block5_preact_relu - False
133 conv4_block5_1_conv - False
134 conv4_block5_1_bn - False
135 conv4_block5_1_relu - False
136 conv4_block5_2_pad - False
137 conv4_block5_2_conv - False
138 conv4_block5_2_bn - False
139 conv4_block5_2_relu - False
140 conv4_block5_3_conv - False
141 conv4_block5_out - False
142 conv4_block6_preact_bn - False
143 conv4_block6_preact_relu - False
144 conv4_block6_1_conv - False
145 conv4_block6_1_bn - False
146 conv4_block6_1_relu - False
147 conv4_block6_2_pad - False
148 conv4_block6_2_conv - False
149 conv4_block6_2_bn - False
150 conv4_block6_2_relu - False
151 max_pooling2d_2 - False
152 conv4_block6_3_conv - False
153 conv4_block6_out - False
154 conv5_block1_preact_bn - False
155 conv5_block1_preact_relu - False
156 conv5_block1_1_conv - False
157 conv5_block1_1_bn - False
158 conv5_block1_1_relu - False
159 conv5_block1_2_pad - False
160 conv5_block1_2_conv - False
161 conv5_block1_2_bn - False
162 conv5_block1_2_relu - False
163 conv5_block1_0_conv - False
164 conv5_block1_3_conv - False
165 conv5_block1_out - False
166 conv5_block2_preact_bn - False
167 conv5_block2_preact_relu - False
168 conv5_block2_1_conv - False
169 conv5_block2_1_bn - False
170 conv5_block2_1_relu - False
171 conv5_block2_2_pad - False
172 conv5_block2_2_conv - False
173 conv5_block2_2_bn - False
174 conv5_block2_2_relu - False
175 conv5_block2_3_conv - False
176 conv5_block2_out - False
177 conv5_block3_preact_bn - False
178 conv5_block3_preact_relu - False
179 conv5_block3_1_conv - False
180 conv5_block3_1_bn - False
181 conv5_block3_1_relu - False
182 conv5_block3_2_pad - False
183 conv5_block3_2_conv - False
184 conv5_block3_2_bn - False
185 conv5_block3_2_relu - False
186 conv5_block3_3_conv - False
187 conv5_block3_out - False
188 post_bn - False
189 post_relu - False
0 input_2 - False
1 Conv1 - False
2 bn_Conv1 - False
3 Conv1_relu - False
4 expanded_conv_depthwise - False
5 expanded_conv_depthwise_BN - False
6 expanded_conv_depthwise_relu - False
7 expanded_conv_project - False
8 expanded_conv_project_BN - False
9 block_1_expand - False
10 block_1_expand_BN - False
11 block_1_expand_relu - False
12 block_1_pad - False
13 block_1_depthwise - False
14 block_1_depthwise_BN - False
15 block_1_depthwise_relu - False
16 block_1_project - False
17 block_1_project_BN - False
18 block_2_expand - False
19 block_2_expand_BN - False
20 block_2_expand_relu - False
21 block_2_depthwise - False
22 block_2_depthwise_BN - False
23 block_2_depthwise_relu - False
24 block_2_project - False
25 block_2_project_BN - False
26 block_2_add - False
27 block_3_expand - False
28 block_3_expand_BN - False
29 block_3_expand_relu - False
30 block_3_pad - False
31 block_3_depthwise - False
32 block_3_depthwise_BN - False
33 block_3_depthwise_relu - False
34 block_3_project - False
35 block_3_project_BN - False
36 block_4_expand - False
37 block_4_expand_BN - False
38 block_4_expand_relu - False
39 block_4_depthwise - False
40 block_4_depthwise_BN - False
41 block_4_depthwise_relu - False
42 block_4_project - False
43 block_4_project_BN - False
44 block_4_add - False
45 block_5_expand - False
46 block_5_expand_BN - False
47 block_5_expand_relu - False
48 block_5_depthwise - False
49 block_5_depthwise_BN - False
50 block_5_depthwise_relu - False
51 block_5_project - False
52 block_5_project_BN - False
53 block_5_add - False
54 block_6_expand - False
55 block_6_expand_BN - False
56 block_6_expand_relu - False
57 block_6_pad - False
58 block_6_depthwise - False
59 block_6_depthwise_BN - False
60 block_6_depthwise_relu - False
61 block_6_project - False
62 block_6_project_BN - False
63 block_7_expand - False
64 block_7_expand_BN - False
65 block_7_expand_relu - False
66 block_7_depthwise - False
67 block_7_depthwise_BN - False
68 block_7_depthwise_relu - False
69 block_7_project - False
70 block_7_project_BN - False
71 block_7_add - False
72 block_8_expand - False
73 block_8_expand_BN - False
74 block_8_expand_relu - False
75 block_8_depthwise - False
76 block_8_depthwise_BN - False
77 block_8_depthwise_relu - False
78 block_8_project - False
79 block_8_project_BN - False
80 block_8_add - False
81 block_9_expand - False
82 block_9_expand_BN - False
83 block_9_expand_relu - False
84 block_9_depthwise - False
85 block_9_depthwise_BN - False
86 block_9_depthwise_relu - False
87 block_9_project - False
88 block_9_project_BN - False
89 block_9_add - False
90 block_10_expand - False
91 block_10_expand_BN - False
92 block_10_expand_relu - False
93 block_10_depthwise - False
94 block_10_depthwise_BN - False
95 block_10_depthwise_relu - False
96 block_10_project - False
97 block_10_project_BN - False
98 block_11_expand - False
99 block_11_expand_BN - False
100 block_11_expand_relu - False
101 block_11_depthwise - False
102 block_11_depthwise_BN - False
103 block_11_depthwise_relu - False
104 block_11_project - False
105 block_11_project_BN - False
106 block_11_add - False
107 block_12_expand - False
108 block_12_expand_BN - False
109 block_12_expand_relu - False
110 block_12_depthwise - False
111 block_12_depthwise_BN - False
112 block_12_depthwise_relu - False
113 block_12_project - False
114 block_12_project_BN - False
115 block_12_add - False
116 block_13_expand - False
117 block_13_expand_BN - False
118 block_13_expand_relu - False
119 block_13_pad - False
120 block_13_depthwise - False
121 block_13_depthwise_BN - False
122 block_13_depthwise_relu - False
123 block_13_project - False
124 block_13_project_BN - False
125 block_14_expand - False
126 block_14_expand_BN - False
127 block_14_expand_relu - False
128 block_14_depthwise - False
129 block_14_depthwise_BN - False
130 block_14_depthwise_relu - False
131 block_14_project - False
132 block_14_project_BN - False
133 block_14_add - False
134 block_15_expand - False
135 block_15_expand_BN - False
136 block_15_expand_relu - False
137 block_15_depthwise - False
138 block_15_depthwise_BN - False
139 block_15_depthwise_relu - False
140 block_15_project - False
141 block_15_project_BN - False
142 block_15_add - False
143 block_16_expand - False
144 block_16_expand_BN - False
145 block_16_expand_relu - False
146 block_16_depthwise - False
147 block_16_depthwise_BN - False
148 block_16_depthwise_relu - False
149 block_16_project - False
150 block_16_project_BN - False
151 Conv_1 - False
152 Conv_1_bn - False
153 out_relu - False
In [45]:
plot_model(res_model, show_shapes=True)
Out[45]:
In [46]:
plot_model(mobile_model, show_shapes=True)
Out[46]:
In [47]:
def build_HMT_teacher_model_fun(show_summary = 0):

    keras.backend.clear_session()
    # Build CNN teacher.
    hmt_transfer_teacher_model = tf.keras.Sequential(name='HMTteacherNET')

    # your code start from here for stpe 2
    hmt_transfer_teacher_model.add(res_model)


    hmt_transfer_teacher_model.add(Flatten())
    hmt_transfer_teacher_model.add(Dense(2,name='logits'))
    hmt_transfer_teacher_model.add(tf.keras.layers.Activation('softmax', name='softmax'))

    for layer in hmt_transfer_teacher_model.layers:
        if(layer.name == 'resnet50v2'):
            layer.trainable = False

    if(show_summary):
        hmt_transfer_teacher_model.summary()

        for i, layer in enumerate(hmt_transfer_teacher_model.layers):
            print(i, layer.name, "-", layer.trainable)
            

    return hmt_transfer_teacher_model

# hmt_transfer_teacher_model = build_HMT_teacher_model_fun(show_summary = 1)

def build_HMT_teacher_model_fun2(show_summary = 0, my_learning_rate = 0.0001):

    keras.backend.clear_session()
    # Build CNN teacher.
    hmt_transfer_teacher_model = tf.keras.Sequential(name='HMTteacherNET')

    # your code start from here for stpe 2
    hmt_transfer_teacher_model.add(res_model)

    hmt_transfer_teacher_model.add(Flatten())
    hmt_transfer_teacher_model.add(Dense(2,name='logits'))
    hmt_transfer_teacher_model.add(tf.keras.layers.Activation('softmax', name='softmax'))

    for layer in hmt_transfer_teacher_model.layers:
        if(layer.name == 'resnet50v2'):
            layer.trainable = False

    my_optimizer=tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)
    my_loss_func=tensorflow.keras.losses.CategoricalCrossentropy()
    hmt_transfer_teacher_model.compile(optimizer=my_optimizer, loss=my_loss_func, metrics=['accuracy'])

    hmt_transfer_teacher_model_logits = tf.keras.Model(inputs=hmt_transfer_teacher_model.input, outputs=hmt_transfer_teacher_model.get_layer('logits').output)

    if(show_summary):
        print('For origin teacher model:')
        hmt_transfer_teacher_model.summary()
        print('\nFor logits teacher model:')
        hmt_transfer_teacher_model_logits.summary()
        for i, layer in enumerate(hmt_transfer_teacher_model.layers):
            print(i, layer.name, "-", layer.trainable)
    

    return hmt_transfer_teacher_model, hmt_transfer_teacher_model_logits

def build_HMT_student_model_fun(show_summary = 0):

    keras.backend.clear_session()
    # Build CNN teacher.
    hmt_student_model = tf.keras.Sequential(name='HMTstudentNET')

    # your code start from here for stpe 2
    hmt_student_model.add(mobile_model)


    hmt_student_model.add(Flatten())
    hmt_student_model.add(Dense(2,name='logits'))
    hmt_student_model.add(tf.keras.layers.Activation('softmax', name='softmax'))

    for layer in hmt_student_model.layers:
        if(layer.name == 'mobilenetv2_1.00_224'):
            layer.trainable = False

    if(show_summary):
        hmt_student_model.summary()

        for i, layer in enumerate(hmt_student_model.layers):
            print(i, layer.name, "-", layer.trainable)

    return hmt_student_model

# hmt_transfer_student_model = build_HMT_student_model_fun(show_summary = 1)

def build_HMT_student_model_fun2(show_summary = 0, my_learning_rate = 0.0001):

    keras.backend.clear_session()
    # Build CNN teacher.
    hmt_student_model = tf.keras.Sequential(name='HMTstudentNET')

    # your code start from here for stpe 2
    hmt_student_model.add(mobile_model)

    hmt_student_model.add(Flatten())
    hmt_student_model.add(Dense(2,name='logits'))
    hmt_student_model.add(tf.keras.layers.Activation('softmax', name='softmax'))

    for layer in hmt_student_model.layers:
        if(layer.name == 'mobilenetv2_1.00_224'):
            layer.trainable = False

    my_optimizer=tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)
    my_loss_func=tensorflow.keras.losses.CategoricalCrossentropy()
    hmt_student_model.compile(optimizer=my_optimizer, loss=my_loss_func, metrics=['accuracy'])
    hmt_student_model_logits = tf.keras.Model(inputs=hmt_student_model.input, outputs=hmt_student_model.get_layer('logits').output)

    if(show_summary):
        print('For origin student model:')
        hmt_student_model.summary()
        print('\nFor logits student model:')
        hmt_student_model_logits.summary()
        for i, layer in enumerate(hmt_student_model.layers):
            print(i, layer.name, "-", layer.trainable)

    return hmt_student_model, hmt_student_model_logits

hmt_transfer_teacher_model, hmt_transfer_teacher_model_logits = build_HMT_teacher_model_fun2(show_summary = 1)
hmt_transfer_student_model, hmt_transfer_student_model_logits = build_HMT_student_model_fun2(show_summary = 1)
For origin teacher model:
Model: "HMTteacherNET"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 resnet50v2 (Functional)     (None, 7, 7, 2048)        23564800  
                                                                 
 flatten (Flatten)           (None, 100352)            0         
                                                                 
 logits (Dense)              (None, 2)                 200706    
                                                                 
 softmax (Activation)        (None, 2)                 0         
                                                                 
=================================================================
Total params: 23,765,506
Trainable params: 200,706
Non-trainable params: 23,564,800
_________________________________________________________________

For logits teacher model:
Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 resnet50v2_input (InputLaye  [(None, 224, 224, 3)]    0         
 r)                                                              
                                                                 
 resnet50v2 (Functional)     (None, 7, 7, 2048)        23564800  
                                                                 
 flatten (Flatten)           (None, 100352)            0         
                                                                 
 logits (Dense)              (None, 2)                 200706    
                                                                 
=================================================================
Total params: 23,765,506
Trainable params: 200,706
Non-trainable params: 23,564,800
_________________________________________________________________
0 resnet50v2 - False
1 flatten - True
2 logits - True
3 softmax - True
For origin student model:
Model: "HMTstudentNET"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 mobilenetv2_1.00_224 (Funct  (None, 7, 7, 1280)       2257984   
 ional)                                                          
                                                                 
 flatten (Flatten)           (None, 62720)             0         
                                                                 
 logits (Dense)              (None, 2)                 125442    
                                                                 
 softmax (Activation)        (None, 2)                 0         
                                                                 
=================================================================
Total params: 2,383,426
Trainable params: 125,442
Non-trainable params: 2,257,984
_________________________________________________________________

For logits student model:
Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 mobilenetv2_1.00_224_input   [(None, 224, 224, 3)]    0         
 (InputLayer)                                                    
                                                                 
 mobilenetv2_1.00_224 (Funct  (None, 7, 7, 1280)       2257984   
 ional)                                                          
                                                                 
 flatten (Flatten)           (None, 62720)             0         
                                                                 
 logits (Dense)              (None, 2)                 125442    
                                                                 
=================================================================
Total params: 2,383,426
Trainable params: 125,442
Non-trainable params: 2,257,984
_________________________________________________________________
0 mobilenetv2_1.00_224 - False
1 flatten - True
2 logits - True
3 softmax - True
In [48]:
# plot_model(hmt_transfer_teacher_model, show_shapes=True)
In [49]:
# plot_model(hmt_transfer_student_model, show_shapes=True)

Teacher loss function

In [13]:
@tf.function
def compute_teacher_loss(images, labels):
    """Compute subclass knowledge distillation teacher loss for given images
        and labels.

    Args:
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

    Returns:
    Scalar loss Tensor.
    """
    subclass_logits = cnn_model(images, training=True)

    # Compute cross-entropy loss for subclasses.

    # your code start from here for step 3
    #   print(images)
    #   print(labels)
    #   one_hot_y = tf.one_hot([labels], 10)
    # one_hot_y = tf.one_hot(labels, 10)
    #   print("one_hot_y is:", one_hot_y)
    #   print("subclass_logits is:", subclass_logits)
    #   print("one_hot_y dim is:", one_hot_y.shape)

    #   diff = subclass_logits-one_hot_y
    y_pred = tf.clip_by_value(subclass_logits, 1e-7, 1 - 1e-7)
    cross_entropy_loss_value = -tf.math.reduce_mean(tf.math.reduce_sum(labels * tf.math.log(y_pred), axis=-1, keepdims=False))

    # cross_entropy_loss_value = 1/2 * (subclass_logits-one_hot_y)**2
    # cross_entropy_loss_value = tf.reduce_sum(cross_entropy_loss_value)

    # print("Current subclass_logits is:", subclass_logits)
    # print("Current labels is:", labels)
    # print("Current cross_entropy_loss_value is:", cross_entropy_loss_value)

    return cross_entropy_loss_value

Student loss function

In [14]:
def softmax_with_temp(logits, temp=1):
    logits = (logits - tf.math.reduce_max(logits)) / temp
    exp_logits = tf.math.exp(logits)
    logits_sum = tf.math.reduce_sum(exp_logits, axis=-1, keepdims=True)
    result = exp_logits / logits_sum
    return result
In [15]:
# #@test {"output": "ignore"}

# Hyperparameters for distillation (need to be tuned).
ALPHA = 0.5 # task balance between cross-entropy and distillation loss
DISTILLATION_TEMPERATURE = 4. #temperature hyperparameter
HMT_BATCH_SIZE = 32
total_train = np_train_set.shape[0]
total_test = np_test_set.shape[0]

def HMT_distillation_loss2(teacher_logits: tf.Tensor, student_logits: tf.Tensor,
                      temperature: Union[float, tf.Tensor]):
  """Compute distillation loss.

  This function computes cross entropy between softened logits and softened
  targets. The resulting loss is scaled by the squared temperature so that
  the gradient magnitude remains approximately constant as the temperature is
  changed. For reference, see Hinton et al., 2014, "Distilling the knowledge in
  a neural network."

  Args:
    teacher_logits: A Tensor of logits provided by the teacher.
    student_logits: A Tensor of logits provided by the student, of the same
      shape as `teacher_logits`.
    temperature: Temperature to use for distillation.

  Returns:
    A scalar Tensor containing the distillation loss.
  """
 # your code start from here for step 3
  soft_targets = softmax_with_temp(teacher_logits,temperature)

  return tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(
          soft_targets, student_logits / temperature)) * temperature ** 2

def HMT_compute_student_loss(images, labels):
    """Compute subclass knowledge distillation student loss for given images
        and labels.

    Args:
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

    Returns:
    Scalar loss Tensor.
    """
    student_subclass_logits_prob = hmt_transfer_student_model(images, training=True)
    student_subclass_logits = hmt_transfer_student_model_logits(images, training=True)

    # Compute subclass distillation loss between student subclass logits and
    # softened teacher subclass targets probabilities.

    # your code start from here for step 3

    teacher_subclass_logits_prob = hmt_transfer_teacher_model(images, training=False)
    teacher_subclass_logits = hmt_transfer_teacher_model_logits(images, training=False)

    distillation_loss_value2 = HMT_distillation_loss2(teacher_subclass_logits, student_subclass_logits, DISTILLATION_TEMPERATURE)

    # Compute cross-entropy loss with hard targets.
    y_pred = tf.clip_by_value(student_subclass_logits_prob, 1e-7, 1 - 1e-7)
    origin_loss = -tf.math.reduce_mean(tf.math.reduce_sum(labels * tf.math.log(y_pred), axis=-1, keepdims=False))
    # your code start from here for step 3

    cross_entropy_loss_value = origin_loss + ALPHA*distillation_loss_value2
    # print("In case of formula 1")
    # weight_cal(origin_loss, ALPHA*distillation_loss_value)
    # print("In case of formula 2")
    # weight_cal(origin_loss, ALPHA*distillation_loss_value2)
    # print("\n")

    return cross_entropy_loss_value

def weight_cal(a,b):
    a = a.numpy()
    b = b.numpy()
    absa = abs(a)
    absb = abs(b)
    print("Origin loss is:", a)
    print("Knowledge distallitation loss is:", b)
    print("Origin loss take %s , and knowledge distallitation take %s weight." % (absa/(absa+absb), absb/(absa+absb)))
    print("And the total loss is:", a+b)
    
    return

Train and evaluation

In [16]:
@tf.function
def HMT_compute_num_correct(model, images, labels):
    """Compute number of correctly classified images in a batch.

    Args:
    model: Instance of tf.keras.Model.
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

    Returns:
    Number of correctly classified images.
    """
    class_logits = model(images, training=False)
    return tf.reduce_sum(
        tf.cast(tf.math.equal(tf.argmax(class_logits, -1), tf.argmax(labels, -1)),
                tf.float32)), tf.argmax(class_logits, -1), tf.argmax(labels, -1)


def HMT_train_and_evaluate(model, compute_loss_fn):
    """Perform training and evaluation for a given model.

    Args:
    model: Instance of tf.keras.Model.
    compute_loss_fn: A function that computes the training loss given the
        images, and labels.
    """

    # your code start from here for step 4
    my_learning_rate = 0.001
    optimizer=tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)

    for epoch in range(1, NUM_EPOCHS + 1):
    # Run training.

        print('Epoch {}: '.format(epoch), end='')

        for i in range(total_train//HMT_BATCH_SIZE+1):
            begin_index = i*HMT_BATCH_SIZE
            end_index = (i+1)*HMT_BATCH_SIZE
            if(i==total_train//HMT_BATCH_SIZE):
                end_index = total_train
            images = np_train_set[begin_index:end_index]
            # images = tf.convert_to_tensor(images)
            labels = np_train_label[begin_index:end_index]
            # labels = tf.convert_to_tensor(labels)
            with tf.GradientTape() as tape:
                # your code start from here for step 4

                loss_value = compute_loss_fn(images, labels)

            grads = tape.gradient(loss_value, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))
            # optimizer.apply_gradients(zip(grads, fc_model_logits.trainable_variables))

        # Run evaluation.
        total_num_correct = 0
        num_total = total_test
        for i in range(total_test//HMT_BATCH_SIZE+1):
            begin_index = i*HMT_BATCH_SIZE
            end_index = (i+1)*HMT_BATCH_SIZE
            if(i==total_train//HMT_BATCH_SIZE):
                end_index = total_train
            images = np_test_set[begin_index:end_index,:,:,:]
            labels = np_test_label[begin_index:end_index,:]
            # your code start from here for step 4
            num_correct,_,_ = compute_num_correct(model, images, labels)
            total_num_correct = total_num_correct + num_correct
        print("Class_accuracy: " + '{:.2f}%'.format(
            total_num_correct / num_total * 100))
    return (total_num_correct / num_total * 100)


@tf.function
def hmt_compute_num_correct(model, images, labels):
  """Compute number of correctly classified images in a batch.

  Args:
    model: Instance of tf.keras.Model.
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

  Returns:
    Number of correctly classified images.
  """
  class_logits = model(images, training=False)
  return tf.reduce_sum(
      tf.cast(tf.math.equal(tf.argmax(class_logits, -1), tf.argmax(labels, -1)),
              tf.float32)), tf.argmax(class_logits, -1), tf.argmax(labels, -1)

my_batch_size = 32
def hmt_train_and_evaluate(model, compute_loss_fn):
    """Perform training and evaluation for a given model.

    Args:
    model: Instance of tf.keras.Model.
    compute_loss_fn: A function that computes the training loss given the
        images, and labels.
    """
    # tf.enable_eager_execution()
    # your code start from here for step 4
    my_learning_rate = 0.001

    # my_learning_rate=0.001/0.1
    optimizer = tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)
    my_batch_size = 15
    NUM_EPOCHS = 12
    for epoch in range(1, NUM_EPOCHS + 1):
        # Run training.
        print('Epoch {}: '.format(epoch), end='')
        x=0
        y=np_train_set.shape[0]//my_batch_size
        for i in range(y):
      
            images=np_train_set[x:x+my_batch_size]
            labels=np_train_label[x:x+my_batch_size]
            # for images, labels in mnist_train:
            with tf.GradientTape() as tape:
                # your code start from here for step 4

                loss_value = compute_loss_fn(images,labels)
                # print('true true')
                # print(loss_value)
                # print(KD_student_model.trainable_variables)
                # loss_value=loss_value.numpy()
            grads = tape.gradient(loss_value, model.trainable_variables)
            # print('gg')

            optimizer.apply_gradients(zip(grads, model.trainable_variables))
            x=x+my_batch_size
        # Run evaluation.
        # num_correct = 0
        # num_total = builder.info.splits['test'].num_examples
        num_total=np_test_set.shape[0]
        total_num_correct=0

    

        xx=0
        # y=np_train_set.shape[0]//my_batch_size
        for i in range(np_test_set.shape[0]//my_batch_size):
            images=np_test_set[xx:xx+my_batch_size]
            labels=np_test_label[xx:xx+my_batch_size]
            # for images, labels in mnist_test:
            # your code start from here for step 4
            # print(compute_num_correct(model,images,labels))
            num_correct,_,_=hmt_compute_num_correct(model,images,labels)
            xx=xx+my_batch_size

            total_num_correct = total_num_correct+num_correct
        print("Class_accuracy: " + '{:.2f}%'.format(
            total_num_correct / num_total * 100))
    
      


    return ((total_num_correct / num_total * 100).numpy())

HMT_BATCH_SIZE = 32
def hmt_train_and_evaluate2(model, compute_loss_fn, NUM_EPOCHS = 12):
    """Perform training and evaluation for a given model.

    Args:
    model: Instance of tf.keras.Model.
    compute_loss_fn: A function that computes the training loss given the
        images, and labels.
    """
    # your code start from here for step 4
    my_learning_rate = 0.0001

    optimizer = tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)

    # NUM_EPOCHS = 12
    for epoch in range(1, NUM_EPOCHS + 1):
        # Run training.
        print('Epoch {}: '.format(epoch), end='')

        for i in range(total_train//HMT_BATCH_SIZE+1):
            begin_index = i*HMT_BATCH_SIZE
            end_index = (i+1)*HMT_BATCH_SIZE
            if(i==total_train//HMT_BATCH_SIZE):
                end_index = total_train
            images = np_train_set[begin_index:end_index]
            labels = np_train_label[begin_index:end_index]
            with tf.GradientTape() as tape:
                # your code start from here for step 4
                loss_value = compute_loss_fn(images,labels)
                # print('Current input image shape is: ', images.shape)
                # print("Current loss value is:", loss_value.numpy())
                # print("Current trainable_variables are: ", model.trainable_variables)
            grads = tape.gradient(loss_value, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))
        num_total=np_test_set.shape[0]
        total_num_correct=0

    

        total_num_correct = 0
        num_total = total_test
        for i in range(total_test//HMT_BATCH_SIZE+1):
            begin_index = i*HMT_BATCH_SIZE
            end_index = (i+1)*HMT_BATCH_SIZE
            if(i==total_test//HMT_BATCH_SIZE):
                end_index = total_test
            images = np_test_set[begin_index:end_index,:,:,:]
            labels = np_test_label[begin_index:end_index,:]
            num_correct,_,_=hmt_compute_num_correct(model,images,labels)
            total_num_correct = total_num_correct+num_correct
        print("Class_accuracy: " + '{:.2f}%'.format(
            total_num_correct / num_total * 100))
    
      


    return ((total_num_correct / num_total * 100).numpy())

HMT_BATCH_SIZE = 32
def hmt_train_and_evaluate3(model, compute_loss_fn, NUM_EPOCHS = 12):
    """Perform training and evaluation for a given model.

    Args:
    model: Instance of tf.keras.Model.
    compute_loss_fn: A function that computes the training loss given the
        images, and labels.
    """
    # your code start from here for step 4
    my_learning_rate = 0.0001

    optimizer = tensorflow.keras.optimizers.Adam(learning_rate=my_learning_rate)

    # NUM_EPOCHS = 12
    for epoch in range(1, NUM_EPOCHS + 1):
        # Run training.
        print('Epoch {}: '.format(epoch), end='')

        for i in range(total_train//HMT_BATCH_SIZE+1):
            begin_index = i*HMT_BATCH_SIZE
            end_index = (i+1)*HMT_BATCH_SIZE
            if(i==total_train//HMT_BATCH_SIZE):
                end_index = total_train
            images = np_train_set[begin_index:end_index]
            labels = np_train_label[begin_index:end_index]
            with tf.GradientTape() as tape:
                # your code start from here for step 4
                loss_value = compute_loss_fn(images,labels)
                # print('Current input image shape is: ', images.shape)
                # print("Current loss value is:", loss_value.numpy())
                # print("Current trainable_variables are: ", model.trainable_variables)
            grads = tape.gradient(loss_value, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))
        num_total=np_test_set.shape[0]
        total_num_correct=0

        num_total = total_test

        num_correct,_,_=hmt_compute_num_correct(model,np_test_set,np_test_label)
        print("Class_accuracy: " + '{:.2f}%'.format(
            num_correct / num_total * 100))

    return ((total_num_correct / num_total * 100).numpy())
In [17]:
def evaluate_accuracy(model, dataset, labelset):
    total_num_correct = 0
    num_total = dataset.shape[0]
    for index in range(dataset.shape[0]):
        input = dataset[index]
        label = np.argmax(labelset[index])
        pred_label = np.argmax(model(np.expand_dims(np.expand_dims(input, axis=0), axis=-1)).numpy())
        if(pred_label == label):
            total_num_correct = total_num_correct + 1
    print("Set_accuracy: " + '{:.2f}%'.format(total_num_correct / num_total * 100))
    return (total_num_correct / num_total * 100)

def evaluate_accuracy2(model, dataset, labelset, batch_size = 32):
    total_num_correct = 0
    num_total = dataset.shape[0]
    for i in range(num_total//batch_size+1):
        begin_index = i*batch_size
        end_index = (i+1)*batch_size
        if(i==num_total//batch_size):
            end_index = num_total
        images = dataset[begin_index:end_index]
        # images = tf.convert_to_tensor(images)
        labels = dataset[begin_index:end_index]
        # labels = tf.convert_to_tensor(labels)
        # your code start from here for step 4
        num_correct,_,_ = compute_num_correct(model, images, labels)
        num_correct = num_correct.np()
        total_num_correct = total_num_correct + num_correct
    acc = total_num_correct / num_total * 100
    return acc

def evaluate_accuracy3(model, dataset, labelset):
    total_num_correct = 0
    num_total = dataset.shape[0]
    num_correct,_,_ = compute_num_correct(model, dataset, labelset)
    acc = num_correct / num_total * 100
    return acc

def compute_num_correct(model, images, labels):
    """Compute number of correctly classified images in a batch.

    Args:
    model: Instance of tf.keras.Model.
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

    Returns:
    Number of correctly classified images.
    """
    class_logits = model(images, training=False)
    return tf.reduce_sum(
        tf.cast(tf.math.equal(tf.argmax(class_logits, -1), tf.argmax(labels, -1)),
                tf.float32)), tf.argmax(class_logits, -1), tf.argmax(labels, -1)

Training models

In [18]:
# your code start from here for step 5 
log_dir2="logs/fit/HMT_teacher" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback2 = tensorflow.keras.callbacks.TensorBoard(log_dir=log_dir2, histogram_freq=1)
In [19]:
hmt_transfer_teacher_model, hmt_transfer_teacher_model_logits = build_HMT_teacher_model_fun2()
hmt_transfer_student_model, hmt_transfer_student_model_logits = build_HMT_student_model_fun2()

history = hmt_transfer_teacher_model.fit(x = np_train_set, y = np_train_label, batch_size = 32, epochs = 6, callbacks=[tensorboard_callback2])
t_acc = evaluate_accuracy3(hmt_transfer_teacher_model, np_test_set, np_test_label)
s_acc = hmt_train_and_evaluate2(hmt_transfer_student_model, HMT_compute_student_loss, NUM_EPOCHS=8)
print('Teacher accuracy is %s and student accuracy is %s.'%(t_acc.numpy(), s_acc))
Epoch 1/6
68/68 [==============================] - 17s 102ms/step - loss: 0.6629 - accuracy: 0.7664
Epoch 2/6
68/68 [==============================] - 7s 97ms/step - loss: 0.1936 - accuracy: 0.9200
Epoch 3/6
68/68 [==============================] - 7s 97ms/step - loss: 0.0756 - accuracy: 0.9830
Epoch 4/6
68/68 [==============================] - 7s 97ms/step - loss: 0.0404 - accuracy: 0.9995
Epoch 5/6
68/68 [==============================] - 7s 97ms/step - loss: 0.0220 - accuracy: 1.0000
Epoch 6/6
68/68 [==============================] - 6s 95ms/step - loss: 0.0157 - accuracy: 1.0000
Epoch 1: Class_accuracy: 58.34%
Epoch 2: Class_accuracy: 74.00%
Epoch 3: Class_accuracy: 74.62%
Epoch 4: Class_accuracy: 74.72%
Epoch 5: Class_accuracy: 73.69%
Epoch 6: Class_accuracy: 73.29%
Epoch 7: Class_accuracy: 73.18%
Epoch 8: Class_accuracy: 73.80%
Teacher accuracy is 79.63152 and student accuracy is 73.79734.
In [20]:
print('Teacher accuracy is %s and student accuracy is %s.'%(t_acc.numpy(), s_acc))
Teacher accuracy is 79.63152 and student accuracy is 73.79734.

Test accuracy vs. tempreture curve

In [ ]:
# your code start from here for step 6
temperature_array = [1.,2.,4.,8.,16.,32.,64.]
class_accuracy_array = []
for t in temperature_array:
    DISTILLATION_TEMPERATURE = t
    hmt_transfer_student_model, hmt_transfer_student_model_logits = build_HMT_student_model_fun2()
    acc = hmt_train_and_evaluate2(hmt_transfer_student_model, HMT_compute_student_loss, NUM_EPOCHS=8)
    print("In temperature = %s, accuracy after 12 epoch is: %s." %(t,acc))
    class_accuracy_array.append(acc)
Epoch 1: Class_accuracy: 36.85%
Epoch 2: Class_accuracy: 50.77%
Epoch 3: Class_accuracy: 47.08%
Epoch 4: Class_accuracy: 52.20%
Epoch 5: Class_accuracy: 54.15%
Epoch 6: Class_accuracy: 55.78%
Epoch 7: Class_accuracy: 57.93%
Epoch 8: Class_accuracy: 59.98%
In temperature = 1.0, accuracy after 12 epoch is: 59.979527.
Epoch 1: Class_accuracy: 36.85%
Epoch 2: Class_accuracy: 36.85%
Epoch 3: Class_accuracy: 43.30%
Epoch 4: Class_accuracy: 47.49%
Epoch 5: Class_accuracy: 50.56%
Epoch 6: Class_accuracy: 52.61%
Epoch 7: Class_accuracy: 54.76%
Epoch 8: Class_accuracy: 54.86%
In temperature = 2.0, accuracy after 12 epoch is: 54.86182.
Epoch 1: Class_accuracy: 69.91%
Epoch 2: Class_accuracy: 72.36%
Epoch 3: Class_accuracy: 73.90%
Epoch 4: Class_accuracy: 74.00%
Epoch 5: Class_accuracy: 72.47%
Epoch 6: Class_accuracy: 72.36%
Epoch 7: Class_accuracy: 72.26%
Epoch 8: Class_accuracy: 72.88%
In temperature = 4.0, accuracy after 12 epoch is: 72.87615.
Epoch 1: Class_accuracy: 45.75%
Epoch 2: Class_accuracy: 73.39%
Epoch 3: Class_accuracy: 73.29%
Epoch 4: Class_accuracy: 72.98%
Epoch 5: Class_accuracy: 71.85%
Epoch 6: Class_accuracy: 71.14%
Epoch 7: Class_accuracy: 70.93%
Epoch 8: Class_accuracy: 71.03%
In temperature = 8.0, accuracy after 12 epoch is: 71.033775.
Epoch 1: Class_accuracy: 53.43%
Epoch 2: Class_accuracy: 70.21%
Epoch 3: Class_accuracy: 72.57%
Epoch 4: Class_accuracy: 71.44%
Epoch 5: Class_accuracy: 70.52%
Epoch 6: Class_accuracy: 68.88%
Epoch 7: Class_accuracy: 68.47%
Epoch 8: Class_accuracy: 68.17%
In temperature = 16.0, accuracy after 12 epoch is: 68.16786.
Epoch 1: Class_accuracy: 44.22%
Epoch 2: Class_accuracy: 72.47%
Epoch 3: Class_accuracy: 72.88%
Epoch 4: Class_accuracy: 72.36%
Epoch 5: Class_accuracy: 70.01%
Epoch 6: Class_accuracy: 69.29%
Epoch 7: Class_accuracy: 68.99%
Epoch 8: Class_accuracy: 69.70%
In temperature = 32.0, accuracy after 12 epoch is: 69.70317.
Epoch 1: Class_accuracy: 54.76%
Epoch 2: Class_accuracy: 70.01%
Epoch 3: Class_accuracy: 73.29%
Epoch 4: Class_accuracy: 72.88%
Epoch 5: Class_accuracy: 71.03%
Epoch 6: Class_accuracy: 70.73%
Epoch 7: Class_accuracy: 70.21%
Epoch 8: Class_accuracy: 69.29%
In temperature = 64.0, accuracy after 12 epoch is: 69.293755.
In [ ]:
class_accuracy_array_np = np.array(class_accuracy_array)
print(class_accuracy_array_np)
plt.plot(temperature_array, class_accuracy_array_np)
plt.title("Accuracy vs temperature")
plt.xlabel('Temperature')
plt.ylabel('Accuracy')
plt.show()
[59.979527 54.86182  72.87615  71.033775 68.16786  69.70317  69.293755]

Train student from scratch

In [21]:
hmt_transfer_student_model_no_distillation, _ = build_HMT_student_model_fun2()
hmt_transfer_student_model_no_distillation.fit(x = np_train_set, y = np_train_label, batch_size = 32, epochs = 8, callbacks=[tensorboard_callback2])
s_wKD_acc = evaluate_accuracy3(hmt_transfer_student_model_no_distillation, np_test_set, np_test_label)
print('Student without KD accuracy is %s.'%(s_wKD_acc.numpy()))
Epoch 1/8
68/68 [==============================] - 5s 48ms/step - loss: 0.6169 - accuracy: 0.7577
Epoch 2/8
68/68 [==============================] - 3s 46ms/step - loss: 0.2495 - accuracy: 0.8952
Epoch 3/8
68/68 [==============================] - 3s 48ms/step - loss: 0.1343 - accuracy: 0.9554
Epoch 4/8
68/68 [==============================] - 3s 47ms/step - loss: 0.0792 - accuracy: 0.9811
Epoch 5/8
68/68 [==============================] - 3s 47ms/step - loss: 0.0469 - accuracy: 0.9977
Epoch 6/8
68/68 [==============================] - 3s 47ms/step - loss: 0.0329 - accuracy: 0.9995
Epoch 7/8
68/68 [==============================] - 3s 47ms/step - loss: 0.0254 - accuracy: 1.0000
Epoch 8/8
68/68 [==============================] - 3s 46ms/step - loss: 0.0191 - accuracy: 1.0000
Student without KD accuracy is 77.6868.
In [34]:
hmt_transfer_student_model_no_distillation2, hmt_transfer_student_model_logits = build_HMT_student_model_fun2()
## @test {"output": "ignore"}

def HMT_compute_plain_cross_entropy_loss(images, labels):
    """Compute plain loss for given images and labels.

    For fair comparison and convenience, this function also performs a
    LogSumExp over subclasses, but does not perform subclass distillation.

    Args:
    images: Tensor representing a batch of images.
    labels: Tensor representing a batch of labels.

    Returns:
    Scalar loss Tensor.
    """
    # your code start from here for step 7

    student_subclass_logits_prob = hmt_transfer_student_model_no_distillation(images, training=True)

    # Compute cross-entropy loss with hard targets.
    y_pred = tf.clip_by_value(student_subclass_logits_prob, 1e-7, 1 - 1e-7)
    origin_loss = -tf.math.reduce_mean(tf.math.reduce_sum(labels * tf.math.log(y_pred), axis=-1, keepdims=False))
    # your code start from here for step 3

    cross_entropy_loss_value = origin_loss

    return cross_entropy_loss_value

ALPHA = 0.0
# s_wKD_acc2 = hmt_train_and_evaluate2(hmt_transfer_student_model_no_distillation2, HMT_compute_plain_cross_entropy_loss, NUM_EPOCHS=8)
s_wKD_acc2 = hmt_train_and_evaluate2(hmt_transfer_student_model_no_distillation2, HMT_compute_student_loss, NUM_EPOCHS=8)
ALPHA = 0.5
print('Student without KD accuracy is %s.'%(s_wKD_acc2))
Epoch 1: Class_accuracy: 44.22%
Epoch 2: Class_accuracy: 44.22%
Epoch 3: Class_accuracy: 44.22%
Epoch 4: Class_accuracy: 44.22%
Epoch 5: Class_accuracy: 44.22%
Epoch 6: Class_accuracy: 44.22%
Epoch 7: Class_accuracy: 44.22%
Epoch 8: Class_accuracy: 44.22%
Student without KD accuracy is 44.21699.

Comparing the teacher and student model (number of of parameters and FLOPs)

In [32]:
# your code start from here for step 8
print("For teacher model:")
flops = get_flops(hmt_transfer_teacher_model, batch_size = 1)
print(f"FLOPS: {flops / 10 ** 9:.03}G")
print("For student model:")
flops = get_flops(hmt_transfer_student_model, batch_size = 1)
print(f"FLOPS: {flops / 10 ** 9:.03}G")
print("For student model without distillation:")
flops = get_flops(hmt_transfer_student_model_no_distillation, batch_size = 1)
print(f"FLOPS: {flops / 10 ** 9:.03}G")
For teacher model:
FLOPS: 6.99G
For student model:
FLOPS: 0.613G
For student model without distillation:
FLOPS: 0.613G

XAI method to explain models

In [35]:
# your code start from here for step 9
%load_ext autoreload
%autoreload 2
from xai_utils import *
In [36]:
def my_HMT_XAI_test(image_batch, label_batch, index, model1 = hmt_transfer_teacher_model, model2 = hmt_transfer_student_model, model3 = hmt_transfer_student_model_no_distillation):

    prediction=model1(image_batch)
    prediction_2=model2(image_batch)
    prediction_3=model3(image_batch)
    print('True label: ',np.argmax(label_batch[index]))
    print('Teacher predicted_label: ',np.argmax(prediction[index]))
    print('Teacher confidence score for the correct label: '+str(prediction[index][np.argmax(label_batch[index])].numpy()))
    print('Student predicted_label: ',np.argmax(prediction_2[index]))
    print('Student confidence score for the correct label: '+str(prediction_2[index][np.argmax(label_batch[index])].numpy()))
    print('Student no distillation predicted_label: ',np.argmax(prediction_3[index]))
    print('Student no distillation confidence score for the correct label: '+str(prediction_3[index][np.argmax(label_batch[index])].numpy()))

    explanation_map_RISE = RISE(image_batch[index], model1, class_index=np.argmax(prediction[index]) ,N_MASKS=1000, H = 224, W = 224, C = 3)
    explanation_map_RISE -= explanation_map_RISE.min()
    explanation_map_RISE /= explanation_map_RISE.max()+10e-30

    explanation_map_RISE_2 = RISE(image_batch[index], model2, class_index=np.argmax(prediction[index]) ,N_MASKS=1000, H = 224, W = 224, C = 3)
    explanation_map_RISE_2 -= explanation_map_RISE_2.min()
    explanation_map_RISE_2 /= explanation_map_RISE_2.max()+10e-30

    explanation_map_RISE_3 = RISE(image_batch[index], model3, class_index=np.argmax(prediction[index]) ,N_MASKS=1000, H = 224, W = 224, C = 3)
    explanation_map_RISE_3 -= explanation_map_RISE_3.min()
    explanation_map_RISE_3 /= explanation_map_RISE_3.max()+10e-30

    plt.figure(figsize=(20,5))

    plt.subplot(1,4,1)
    plt.imshow(image_batch[index,:,:,0])
    plt.axis('off')
    plt.title('Sample image')

    plt.subplot(1,4,2)
    plt.imshow(image_batch[index,:,:,0])
    plt.imshow(explanation_map_RISE, cmap='jet', alpha=0.5)
    plt.axis('off')
    plt.title('Explanation map (RISE) for teacher model')

    plt.subplot(1,4,3)
    plt.imshow(image_batch[index,:,:,0])
    plt.imshow(explanation_map_RISE_2, cmap='jet', alpha=0.5)
    plt.axis('off')
    plt.title('Explanation map (RISE) for student model')

    plt.subplot(1,4,4)
    plt.imshow(image_batch[index,:,:,0])
    plt.imshow(explanation_map_RISE_3, cmap='jet', alpha=0.5)
    plt.axis('off')
    plt.title('Explanation map (RISE) for student model no distillation')

my_HMT_XAI_test(np_train_set[0:10,:,:,:], np_train_label[0:10,:], 3)
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.99919003
Student predicted_label:  1
Student confidence score for the correct label: 0.9313379
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.997391
In [37]:
for index in range(5):
    my_HMT_XAI_test(np_train_set[100:110,:,:,:], np_train_label[100:110,:], index)
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.99984527
Student predicted_label:  1
Student confidence score for the correct label: 0.9888342
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.9963217
True label:  0
Teacher predicted_label:  0
Teacher confidence score for the correct label: 0.99994063
Student predicted_label:  0
Student confidence score for the correct label: 0.99983895
Student no distillation predicted_label:  0
Student no distillation confidence score for the correct label: 0.99990237
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.90778226
Student predicted_label:  0
Student confidence score for the correct label: 0.03808604
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.94336426
True label:  0
Teacher predicted_label:  0
Teacher confidence score for the correct label: 0.9968786
Student predicted_label:  0
Student confidence score for the correct label: 0.93169034
Student no distillation predicted_label:  0
Student no distillation confidence score for the correct label: 0.9509316
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.990546
Student predicted_label:  0
Student confidence score for the correct label: 0.00034703585
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.9824934
In [38]:
for index in range(5):
    my_HMT_XAI_test(np_train_set[200:210,:,:,:], np_train_label[200:210,:], index)
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.97148246
Student predicted_label:  1
Student confidence score for the correct label: 0.98983496
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.9906517
True label:  0
Teacher predicted_label:  0
Teacher confidence score for the correct label: 0.99759895
Student predicted_label:  0
Student confidence score for the correct label: 0.880637
Student no distillation predicted_label:  0
Student no distillation confidence score for the correct label: 0.9973152
True label:  0
Teacher predicted_label:  0
Teacher confidence score for the correct label: 0.9999924
Student predicted_label:  0
Student confidence score for the correct label: 0.9999882
Student no distillation predicted_label:  0
Student no distillation confidence score for the correct label: 0.9999722
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.9912368
Student predicted_label:  0
Student confidence score for the correct label: 0.035518743
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.9938379
True label:  1
Teacher predicted_label:  1
Teacher confidence score for the correct label: 0.9859147
Student predicted_label:  1
Student confidence score for the correct label: 0.9500885
Student no distillation predicted_label:  1
Student no distillation confidence score for the correct label: 0.99935275

Implementing the state-of-the-art KD algorithm

In [39]:
epoch_array = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]

hmt_transfer_teacher_model_acc = []
hmt_transfer_student_model_acc = []

for epoch_num in epoch_array:
    hmt_transfer_teacher_model, hmt_transfer_teacher_model_logits = build_HMT_teacher_model_fun2()
    hmt_transfer_student_model, hmt_transfer_student_model_logits = build_HMT_student_model_fun2()

    history = hmt_transfer_teacher_model.fit(x = np_train_set, y = np_train_label, batch_size = 32, epochs = epoch_num, callbacks=[tensorboard_callback2])
    teacher_test_acc = evaluate_accuracy3(hmt_transfer_teacher_model, np_test_set, np_test_label)
    student_test_acc = hmt_train_and_evaluate2(hmt_transfer_student_model, HMT_compute_student_loss, NUM_EPOCHS=8)
    hmt_transfer_teacher_model_acc.append(teacher_test_acc)
    hmt_transfer_student_model_acc.append(student_test_acc)
68/68 [==============================] - 9s 93ms/step - loss: 0.7105 - accuracy: 0.7457
Epoch 1: Class_accuracy: 77.48%
Epoch 2: Class_accuracy: 78.10%
Epoch 3: Class_accuracy: 75.74%
Epoch 4: Class_accuracy: 73.29%
Epoch 5: Class_accuracy: 75.64%
Epoch 6: Class_accuracy: 74.31%
Epoch 7: Class_accuracy: 73.18%
Epoch 8: Class_accuracy: 68.37%
Epoch 1/2
68/68 [==============================] - 9s 93ms/step - loss: 0.6959 - accuracy: 0.7411
Epoch 2/2
68/68 [==============================] - 6s 93ms/step - loss: 0.1964 - accuracy: 0.9228
Epoch 1: Class_accuracy: 76.56%
Epoch 2: Class_accuracy: 76.66%
Epoch 3: Class_accuracy: 76.56%
Epoch 4: Class_accuracy: 76.97%
Epoch 5: Class_accuracy: 70.83%
Epoch 6: Class_accuracy: 78.20%
Epoch 7: Class_accuracy: 74.41%
Epoch 8: Class_accuracy: 73.18%
Epoch 1/3
68/68 [==============================] - 9s 93ms/step - loss: 0.6551 - accuracy: 0.7600
Epoch 2/3
68/68 [==============================] - 6s 93ms/step - loss: 0.1675 - accuracy: 0.9324
Epoch 3/3
68/68 [==============================] - 6s 93ms/step - loss: 0.0657 - accuracy: 0.9862
Epoch 1: Class_accuracy: 75.64%
Epoch 2: Class_accuracy: 75.74%
Epoch 3: Class_accuracy: 76.15%
Epoch 4: Class_accuracy: 77.38%
Epoch 5: Class_accuracy: 76.77%
Epoch 6: Class_accuracy: 70.83%
Epoch 7: Class_accuracy: 70.21%
Epoch 8: Class_accuracy: 77.58%
Epoch 1/4
68/68 [==============================] - 9s 93ms/step - loss: 0.6738 - accuracy: 0.7632
Epoch 2/4
68/68 [==============================] - 6s 93ms/step - loss: 0.2011 - accuracy: 0.9177
Epoch 3/4
68/68 [==============================] - 6s 92ms/step - loss: 0.0631 - accuracy: 0.9848
Epoch 4/4
68/68 [==============================] - 6s 92ms/step - loss: 0.0335 - accuracy: 0.9986
Epoch 1: Class_accuracy: 74.62%
Epoch 2: Class_accuracy: 74.51%
Epoch 3: Class_accuracy: 73.18%
Epoch 4: Class_accuracy: 74.82%
Epoch 5: Class_accuracy: 77.99%
Epoch 6: Class_accuracy: 76.36%
Epoch 7: Class_accuracy: 68.47%
Epoch 8: Class_accuracy: 67.04%
Epoch 1/5
68/68 [==============================] - 9s 94ms/step - loss: 0.7501 - accuracy: 0.7467
Epoch 2/5
68/68 [==============================] - 6s 92ms/step - loss: 0.2176 - accuracy: 0.9085
Epoch 3/5
68/68 [==============================] - 6s 93ms/step - loss: 0.0785 - accuracy: 0.9807
Epoch 4/5
68/68 [==============================] - 6s 93ms/step - loss: 0.0384 - accuracy: 0.9986
Epoch 5/5
68/68 [==============================] - 6s 93ms/step - loss: 0.0235 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.03%
Epoch 2: Class_accuracy: 76.15%
Epoch 3: Class_accuracy: 76.97%
Epoch 4: Class_accuracy: 76.97%
Epoch 5: Class_accuracy: 76.66%
Epoch 6: Class_accuracy: 74.31%
Epoch 7: Class_accuracy: 64.89%
Epoch 8: Class_accuracy: 67.14%
Epoch 1/6
68/68 [==============================] - 9s 92ms/step - loss: 0.7457 - accuracy: 0.7531
Epoch 2/6
68/68 [==============================] - 6s 93ms/step - loss: 0.2092 - accuracy: 0.9200
Epoch 3/6
68/68 [==============================] - 6s 93ms/step - loss: 0.0800 - accuracy: 0.9816
Epoch 4/6
68/68 [==============================] - 6s 93ms/step - loss: 0.0405 - accuracy: 0.9968
Epoch 5/6
68/68 [==============================] - 6s 92ms/step - loss: 0.0235 - accuracy: 0.9995
Epoch 6/6
68/68 [==============================] - 6s 94ms/step - loss: 0.0171 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.46%
Epoch 2: Class_accuracy: 77.38%
Epoch 3: Class_accuracy: 77.79%
Epoch 4: Class_accuracy: 76.46%
Epoch 5: Class_accuracy: 68.99%
Epoch 6: Class_accuracy: 78.61%
Epoch 7: Class_accuracy: 74.62%
Epoch 8: Class_accuracy: 77.07%
Epoch 1/7
68/68 [==============================] - 9s 93ms/step - loss: 0.7002 - accuracy: 0.7522
Epoch 2/7
68/68 [==============================] - 6s 93ms/step - loss: 0.1931 - accuracy: 0.9241
Epoch 3/7
68/68 [==============================] - 6s 93ms/step - loss: 0.0753 - accuracy: 0.9807
Epoch 4/7
68/68 [==============================] - 6s 93ms/step - loss: 0.0400 - accuracy: 0.9982
Epoch 5/7
68/68 [==============================] - 6s 93ms/step - loss: 0.0229 - accuracy: 0.9995
Epoch 6/7
68/68 [==============================] - 6s 92ms/step - loss: 0.0165 - accuracy: 1.0000
Epoch 7/7
68/68 [==============================] - 6s 93ms/step - loss: 0.0131 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.74%
Epoch 2: Class_accuracy: 76.87%
Epoch 3: Class_accuracy: 73.59%
Epoch 4: Class_accuracy: 76.15%
Epoch 5: Class_accuracy: 76.87%
Epoch 6: Class_accuracy: 78.51%
Epoch 7: Class_accuracy: 69.19%
Epoch 8: Class_accuracy: 68.27%
Epoch 1/8
68/68 [==============================] - 9s 93ms/step - loss: 0.6340 - accuracy: 0.7701
Epoch 2/8
68/68 [==============================] - 6s 94ms/step - loss: 0.1828 - accuracy: 0.9278
Epoch 3/8
68/68 [==============================] - 6s 92ms/step - loss: 0.0652 - accuracy: 0.9853
Epoch 4/8
68/68 [==============================] - 6s 92ms/step - loss: 0.0330 - accuracy: 0.9991
Epoch 5/8
68/68 [==============================] - 7s 103ms/step - loss: 0.0205 - accuracy: 1.0000
Epoch 6/8
68/68 [==============================] - 9s 131ms/step - loss: 0.0154 - accuracy: 1.0000
Epoch 7/8
68/68 [==============================] - 6s 96ms/step - loss: 0.0120 - accuracy: 1.0000
Epoch 8/8
68/68 [==============================] - 6s 94ms/step - loss: 0.0099 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.46%
Epoch 2: Class_accuracy: 77.07%
Epoch 3: Class_accuracy: 77.58%
Epoch 4: Class_accuracy: 77.79%
Epoch 5: Class_accuracy: 76.46%
Epoch 6: Class_accuracy: 77.99%
Epoch 7: Class_accuracy: 70.93%
Epoch 8: Class_accuracy: 71.95%
Epoch 1/9
68/68 [==============================] - 9s 93ms/step - loss: 0.7058 - accuracy: 0.7632
Epoch 2/9
68/68 [==============================] - 6s 93ms/step - loss: 0.2075 - accuracy: 0.9209
Epoch 3/9
68/68 [==============================] - 6s 93ms/step - loss: 0.0802 - accuracy: 0.9821
Epoch 4/9
68/68 [==============================] - 6s 92ms/step - loss: 0.0362 - accuracy: 0.9972
Epoch 5/9
68/68 [==============================] - 6s 92ms/step - loss: 0.0225 - accuracy: 1.0000
Epoch 6/9
68/68 [==============================] - 6s 93ms/step - loss: 0.0161 - accuracy: 1.0000
Epoch 7/9
68/68 [==============================] - 6s 93ms/step - loss: 0.0126 - accuracy: 1.0000
Epoch 8/9
68/68 [==============================] - 6s 93ms/step - loss: 0.0103 - accuracy: 1.0000
Epoch 9/9
68/68 [==============================] - 6s 93ms/step - loss: 0.0088 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.64%
Epoch 2: Class_accuracy: 77.48%
Epoch 3: Class_accuracy: 77.79%
Epoch 4: Class_accuracy: 76.36%
Epoch 5: Class_accuracy: 74.62%
Epoch 6: Class_accuracy: 72.26%
Epoch 7: Class_accuracy: 69.60%
Epoch 8: Class_accuracy: 66.33%
Epoch 1/10
68/68 [==============================] - 9s 93ms/step - loss: 0.6826 - accuracy: 0.7508
Epoch 2/10
68/68 [==============================] - 6s 93ms/step - loss: 0.1894 - accuracy: 0.9269
Epoch 3/10
68/68 [==============================] - 6s 93ms/step - loss: 0.0706 - accuracy: 0.9848
Epoch 4/10
68/68 [==============================] - 6s 92ms/step - loss: 0.0344 - accuracy: 0.9995
Epoch 5/10
68/68 [==============================] - 6s 93ms/step - loss: 0.0221 - accuracy: 1.0000
Epoch 6/10
68/68 [==============================] - 6s 92ms/step - loss: 0.0167 - accuracy: 1.0000
Epoch 7/10
68/68 [==============================] - 6s 93ms/step - loss: 0.0130 - accuracy: 1.0000
Epoch 8/10
68/68 [==============================] - 6s 94ms/step - loss: 0.0105 - accuracy: 1.0000
Epoch 9/10
68/68 [==============================] - 6s 93ms/step - loss: 0.0087 - accuracy: 1.0000
Epoch 10/10
68/68 [==============================] - 6s 92ms/step - loss: 0.0075 - accuracy: 1.0000
Epoch 1: Class_accuracy: 79.63%
Epoch 2: Class_accuracy: 79.94%
Epoch 3: Class_accuracy: 78.30%
Epoch 4: Class_accuracy: 77.89%
Epoch 5: Class_accuracy: 78.71%
Epoch 6: Class_accuracy: 67.76%
Epoch 7: Class_accuracy: 66.63%
Epoch 8: Class_accuracy: 70.42%
Epoch 1/11
68/68 [==============================] - 9s 94ms/step - loss: 0.7229 - accuracy: 0.7540
Epoch 2/11
68/68 [==============================] - 6s 92ms/step - loss: 0.2432 - accuracy: 0.8952
Epoch 3/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0738 - accuracy: 0.9811
Epoch 4/11
68/68 [==============================] - 6s 94ms/step - loss: 0.0363 - accuracy: 0.9982
Epoch 5/11
68/68 [==============================] - 6s 94ms/step - loss: 0.0224 - accuracy: 1.0000
Epoch 6/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0166 - accuracy: 1.0000
Epoch 7/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0132 - accuracy: 1.0000
Epoch 8/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0108 - accuracy: 1.0000
Epoch 9/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0091 - accuracy: 1.0000
Epoch 10/11
68/68 [==============================] - 6s 93ms/step - loss: 0.0078 - accuracy: 1.0000
Epoch 11/11
68/68 [==============================] - 6s 94ms/step - loss: 0.0068 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.25%
Epoch 2: Class_accuracy: 77.69%
Epoch 3: Class_accuracy: 79.22%
Epoch 4: Class_accuracy: 78.30%
Epoch 5: Class_accuracy: 77.48%
Epoch 6: Class_accuracy: 77.79%
Epoch 7: Class_accuracy: 76.77%
Epoch 8: Class_accuracy: 66.02%
Epoch 1/12
68/68 [==============================] - 9s 93ms/step - loss: 0.7490 - accuracy: 0.7411
Epoch 2/12
68/68 [==============================] - 6s 93ms/step - loss: 0.2431 - accuracy: 0.9030
Epoch 3/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0820 - accuracy: 0.9752
Epoch 4/12
68/68 [==============================] - 6s 94ms/step - loss: 0.0384 - accuracy: 0.9986
Epoch 5/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0241 - accuracy: 1.0000
Epoch 6/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0177 - accuracy: 1.0000
Epoch 7/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0138 - accuracy: 1.0000
Epoch 8/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0112 - accuracy: 1.0000
Epoch 9/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0094 - accuracy: 1.0000
Epoch 10/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0080 - accuracy: 1.0000
Epoch 11/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0071 - accuracy: 1.0000
Epoch 12/12
68/68 [==============================] - 6s 93ms/step - loss: 0.0061 - accuracy: 1.0000
Epoch 1: Class_accuracy: 77.48%
Epoch 2: Class_accuracy: 75.23%
Epoch 3: Class_accuracy: 74.31%
Epoch 4: Class_accuracy: 77.28%
Epoch 5: Class_accuracy: 77.58%
Epoch 6: Class_accuracy: 77.38%
Epoch 7: Class_accuracy: 69.29%
Epoch 8: Class_accuracy: 68.47%
Epoch 1/13
68/68 [==============================] - 9s 94ms/step - loss: 0.6481 - accuracy: 0.7600
Epoch 2/13
68/68 [==============================] - 6s 94ms/step - loss: 0.1757 - accuracy: 0.9301
Epoch 3/13
68/68 [==============================] - 6s 95ms/step - loss: 0.0679 - accuracy: 0.9871
Epoch 4/13
68/68 [==============================] - 6s 93ms/step - loss: 0.0327 - accuracy: 0.9986
Epoch 5/13
68/68 [==============================] - 6s 93ms/step - loss: 0.0206 - accuracy: 1.0000
Epoch 6/13
68/68 [==============================] - 6s 93ms/step - loss: 0.0151 - accuracy: 1.0000
Epoch 7/13
68/68 [==============================] - 6s 93ms/step - loss: 0.0121 - accuracy: 1.0000
Epoch 8/13
68/68 [==============================] - 6s 94ms/step - loss: 0.0098 - accuracy: 1.0000
Epoch 9/13
68/68 [==============================] - 6s 94ms/step - loss: 0.0082 - accuracy: 1.0000
Epoch 10/13
68/68 [==============================] - 6s 94ms/step - loss: 0.0071 - accuracy: 1.0000
Epoch 11/13
68/68 [==============================] - 6s 93ms/step - loss: 0.0061 - accuracy: 1.0000
Epoch 12/13
68/68 [==============================] - 6s 94ms/step - loss: 0.0054 - accuracy: 1.0000
Epoch 13/13
68/68 [==============================] - 6s 94ms/step - loss: 0.0047 - accuracy: 1.0000
Epoch 1: Class_accuracy: 74.82%
Epoch 2: Class_accuracy: 76.56%
Epoch 3: Class_accuracy: 75.54%
Epoch 4: Class_accuracy: 77.48%
Epoch 5: Class_accuracy: 77.58%
Epoch 6: Class_accuracy: 77.99%
Epoch 7: Class_accuracy: 69.09%
Epoch 8: Class_accuracy: 67.96%
Epoch 1/14
68/68 [==============================] - 9s 94ms/step - loss: 0.6577 - accuracy: 0.7618
Epoch 2/14
68/68 [==============================] - 6s 94ms/step - loss: 0.1876 - accuracy: 0.9264
Epoch 3/14
68/68 [==============================] - 6s 93ms/step - loss: 0.0752 - accuracy: 0.9844
Epoch 4/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0376 - accuracy: 0.9982
Epoch 5/14
68/68 [==============================] - 6s 95ms/step - loss: 0.0228 - accuracy: 1.0000
Epoch 6/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0169 - accuracy: 1.0000
Epoch 7/14
68/68 [==============================] - 6s 93ms/step - loss: 0.0131 - accuracy: 1.0000
Epoch 8/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0103 - accuracy: 1.0000
Epoch 9/14
68/68 [==============================] - 6s 93ms/step - loss: 0.0088 - accuracy: 1.0000
Epoch 10/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0075 - accuracy: 1.0000
Epoch 11/14
68/68 [==============================] - 6s 93ms/step - loss: 0.0065 - accuracy: 1.0000
Epoch 12/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0057 - accuracy: 1.0000
Epoch 13/14
68/68 [==============================] - 6s 94ms/step - loss: 0.0050 - accuracy: 1.0000
Epoch 14/14
68/68 [==============================] - 6s 93ms/step - loss: 0.0044 - accuracy: 1.0000
Epoch 1: Class_accuracy: 73.59%
Epoch 2: Class_accuracy: 74.72%
Epoch 3: Class_accuracy: 79.22%
Epoch 4: Class_accuracy: 77.48%
Epoch 5: Class_accuracy: 76.77%
Epoch 6: Class_accuracy: 68.17%
Epoch 7: Class_accuracy: 68.17%
Epoch 8: Class_accuracy: 71.95%
Epoch 1/15
68/68 [==============================] - 9s 94ms/step - loss: 0.6787 - accuracy: 0.7549
Epoch 2/15
68/68 [==============================] - 6s 94ms/step - loss: 0.1906 - accuracy: 0.9283
Epoch 3/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0662 - accuracy: 0.9876
Epoch 4/15
68/68 [==============================] - 6s 93ms/step - loss: 0.0338 - accuracy: 0.9977
Epoch 5/15
68/68 [==============================] - 6s 93ms/step - loss: 0.0215 - accuracy: 1.0000
Epoch 6/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0157 - accuracy: 1.0000
Epoch 7/15
68/68 [==============================] - 6s 93ms/step - loss: 0.0125 - accuracy: 1.0000
Epoch 8/15
68/68 [==============================] - 6s 95ms/step - loss: 0.0102 - accuracy: 1.0000
Epoch 9/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0085 - accuracy: 1.0000
Epoch 10/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0072 - accuracy: 1.0000
Epoch 11/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0063 - accuracy: 1.0000
Epoch 12/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0055 - accuracy: 1.0000
Epoch 13/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0048 - accuracy: 1.0000
Epoch 14/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0043 - accuracy: 1.0000
Epoch 15/15
68/68 [==============================] - 6s 94ms/step - loss: 0.0039 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.05%
Epoch 2: Class_accuracy: 76.97%
Epoch 3: Class_accuracy: 78.30%
Epoch 4: Class_accuracy: 79.94%
Epoch 5: Class_accuracy: 77.48%
Epoch 6: Class_accuracy: 75.64%
Epoch 7: Class_accuracy: 67.66%
Epoch 8: Class_accuracy: 68.99%
Epoch 1/16
68/68 [==============================] - 9s 94ms/step - loss: 0.6850 - accuracy: 0.7582
Epoch 2/16
68/68 [==============================] - 6s 93ms/step - loss: 0.1981 - accuracy: 0.9269
Epoch 3/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0746 - accuracy: 0.9839
Epoch 4/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0367 - accuracy: 0.9986
Epoch 5/16
68/68 [==============================] - 6s 92ms/step - loss: 0.0225 - accuracy: 1.0000
Epoch 6/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0165 - accuracy: 1.0000
Epoch 7/16
68/68 [==============================] - 6s 92ms/step - loss: 0.0129 - accuracy: 1.0000
Epoch 8/16
68/68 [==============================] - 6s 92ms/step - loss: 0.0105 - accuracy: 1.0000
Epoch 9/16
68/68 [==============================] - 6s 92ms/step - loss: 0.0088 - accuracy: 1.0000
Epoch 10/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0075 - accuracy: 1.0000
Epoch 11/16
68/68 [==============================] - 6s 92ms/step - loss: 0.0065 - accuracy: 1.0000
Epoch 12/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0057 - accuracy: 1.0000
Epoch 13/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0051 - accuracy: 1.0000
Epoch 14/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0044 - accuracy: 1.0000
Epoch 15/16
68/68 [==============================] - 6s 93ms/step - loss: 0.0040 - accuracy: 1.0000
Epoch 16/16
68/68 [==============================] - 6s 94ms/step - loss: 0.0036 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.03%
Epoch 2: Class_accuracy: 76.05%
Epoch 3: Class_accuracy: 77.69%
Epoch 4: Class_accuracy: 77.28%
Epoch 5: Class_accuracy: 74.51%
Epoch 6: Class_accuracy: 66.84%
Epoch 7: Class_accuracy: 66.94%
Epoch 8: Class_accuracy: 71.34%
Epoch 1/17
68/68 [==============================] - 9s 93ms/step - loss: 0.6978 - accuracy: 0.7494
Epoch 2/17
68/68 [==============================] - 6s 93ms/step - loss: 0.2050 - accuracy: 0.9214
Epoch 3/17
68/68 [==============================] - 6s 94ms/step - loss: 0.0740 - accuracy: 0.9839
Epoch 4/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0345 - accuracy: 0.9991
Epoch 5/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0222 - accuracy: 1.0000
Epoch 6/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0168 - accuracy: 1.0000
Epoch 7/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0130 - accuracy: 1.0000
Epoch 8/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0108 - accuracy: 1.0000
Epoch 9/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0090 - accuracy: 1.0000
Epoch 10/17
68/68 [==============================] - 6s 94ms/step - loss: 0.0077 - accuracy: 1.0000
Epoch 11/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0066 - accuracy: 1.0000
Epoch 12/17
68/68 [==============================] - 6s 92ms/step - loss: 0.0059 - accuracy: 1.0000
Epoch 13/17
68/68 [==============================] - 6s 94ms/step - loss: 0.0051 - accuracy: 1.0000
Epoch 14/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0045 - accuracy: 1.0000
Epoch 15/17
68/68 [==============================] - 6s 94ms/step - loss: 0.0041 - accuracy: 1.0000
Epoch 16/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0037 - accuracy: 1.0000
Epoch 17/17
68/68 [==============================] - 6s 93ms/step - loss: 0.0033 - accuracy: 1.0000
Epoch 1: Class_accuracy: 78.40%
Epoch 2: Class_accuracy: 80.96%
Epoch 3: Class_accuracy: 79.32%
Epoch 4: Class_accuracy: 79.22%
Epoch 5: Class_accuracy: 74.92%
Epoch 6: Class_accuracy: 67.45%
Epoch 7: Class_accuracy: 68.37%
Epoch 8: Class_accuracy: 73.39%
Epoch 1/18
68/68 [==============================] - 9s 94ms/step - loss: 0.7106 - accuracy: 0.7554
Epoch 2/18
68/68 [==============================] - 6s 95ms/step - loss: 0.1943 - accuracy: 0.9246
Epoch 3/18
68/68 [==============================] - 6s 95ms/step - loss: 0.0685 - accuracy: 0.9922
Epoch 4/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0361 - accuracy: 0.9986
Epoch 5/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0240 - accuracy: 1.0000
Epoch 6/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0177 - accuracy: 1.0000
Epoch 7/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0136 - accuracy: 1.0000
Epoch 8/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0111 - accuracy: 1.0000
Epoch 9/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0093 - accuracy: 1.0000
Epoch 10/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0080 - accuracy: 1.0000
Epoch 11/18
68/68 [==============================] - 6s 94ms/step - loss: 0.0067 - accuracy: 1.0000
Epoch 12/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0059 - accuracy: 1.0000
Epoch 13/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0052 - accuracy: 1.0000
Epoch 14/18
68/68 [==============================] - 6s 92ms/step - loss: 0.0046 - accuracy: 1.0000
Epoch 15/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0041 - accuracy: 1.0000
Epoch 16/18
68/68 [==============================] - 6s 92ms/step - loss: 0.0037 - accuracy: 1.0000
Epoch 17/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 18/18
68/68 [==============================] - 6s 93ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.44%
Epoch 2: Class_accuracy: 76.56%
Epoch 3: Class_accuracy: 77.38%
Epoch 4: Class_accuracy: 79.02%
Epoch 5: Class_accuracy: 78.10%
Epoch 6: Class_accuracy: 77.99%
Epoch 7: Class_accuracy: 69.70%
Epoch 8: Class_accuracy: 69.19%
Epoch 1/19
68/68 [==============================] - 9s 92ms/step - loss: 0.6419 - accuracy: 0.7605
Epoch 2/19
68/68 [==============================] - 6s 93ms/step - loss: 0.1791 - accuracy: 0.9251
Epoch 3/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0693 - accuracy: 0.9853
Epoch 4/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0359 - accuracy: 0.9977
Epoch 5/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0214 - accuracy: 1.0000
Epoch 6/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0154 - accuracy: 1.0000
Epoch 7/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0122 - accuracy: 1.0000
Epoch 8/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0099 - accuracy: 1.0000
Epoch 9/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0085 - accuracy: 1.0000
Epoch 10/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0072 - accuracy: 1.0000
Epoch 11/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0062 - accuracy: 1.0000
Epoch 12/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0054 - accuracy: 1.0000
Epoch 13/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0048 - accuracy: 1.0000
Epoch 14/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0043 - accuracy: 1.0000
Epoch 15/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0039 - accuracy: 1.0000
Epoch 16/19
68/68 [==============================] - 6s 94ms/step - loss: 0.0035 - accuracy: 1.0000
Epoch 17/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 18/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0029 - accuracy: 1.0000
Epoch 19/19
68/68 [==============================] - 6s 93ms/step - loss: 0.0026 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.44%
Epoch 2: Class_accuracy: 76.56%
Epoch 3: Class_accuracy: 77.99%
Epoch 4: Class_accuracy: 77.79%
Epoch 5: Class_accuracy: 78.10%
Epoch 6: Class_accuracy: 71.44%
Epoch 7: Class_accuracy: 65.30%
Epoch 8: Class_accuracy: 67.55%
Epoch 1/20
68/68 [==============================] - 9s 93ms/step - loss: 0.6854 - accuracy: 0.7545
Epoch 2/20
68/68 [==============================] - 6s 93ms/step - loss: 0.1838 - accuracy: 0.9255
Epoch 3/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0667 - accuracy: 0.9871
Epoch 4/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0321 - accuracy: 0.9972
Epoch 5/20
68/68 [==============================] - 6s 94ms/step - loss: 0.0223 - accuracy: 1.0000
Epoch 6/20
68/68 [==============================] - 6s 92ms/step - loss: 0.0154 - accuracy: 1.0000
Epoch 7/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0121 - accuracy: 1.0000
Epoch 8/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0098 - accuracy: 1.0000
Epoch 9/20
68/68 [==============================] - 6s 94ms/step - loss: 0.0082 - accuracy: 1.0000
Epoch 10/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0071 - accuracy: 1.0000
Epoch 11/20
68/68 [==============================] - 6s 92ms/step - loss: 0.0061 - accuracy: 1.0000
Epoch 12/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0053 - accuracy: 1.0000
Epoch 13/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0048 - accuracy: 1.0000
Epoch 14/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0042 - accuracy: 1.0000
Epoch 15/20
68/68 [==============================] - 6s 94ms/step - loss: 0.0038 - accuracy: 1.0000
Epoch 16/20
68/68 [==============================] - 6s 92ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 17/20
68/68 [==============================] - 6s 92ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 18/20
68/68 [==============================] - 6s 94ms/step - loss: 0.0028 - accuracy: 1.0000
Epoch 19/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0026 - accuracy: 1.0000
Epoch 20/20
68/68 [==============================] - 6s 93ms/step - loss: 0.0024 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.74%
Epoch 2: Class_accuracy: 76.97%
Epoch 3: Class_accuracy: 78.61%
Epoch 4: Class_accuracy: 79.32%
Epoch 5: Class_accuracy: 77.28%
Epoch 6: Class_accuracy: 68.78%
Epoch 7: Class_accuracy: 65.30%
Epoch 8: Class_accuracy: 70.21%
Epoch 1/21
68/68 [==============================] - 9s 93ms/step - loss: 0.7308 - accuracy: 0.7411
Epoch 2/21
68/68 [==============================] - 6s 92ms/step - loss: 0.2228 - accuracy: 0.9094
Epoch 3/21
68/68 [==============================] - 6s 94ms/step - loss: 0.0843 - accuracy: 0.9798
Epoch 4/21
68/68 [==============================] - 6s 94ms/step - loss: 0.0380 - accuracy: 0.9972
Epoch 5/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0244 - accuracy: 1.0000
Epoch 6/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0174 - accuracy: 1.0000
Epoch 7/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0135 - accuracy: 1.0000
Epoch 8/21
68/68 [==============================] - 6s 94ms/step - loss: 0.0110 - accuracy: 1.0000
Epoch 9/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0092 - accuracy: 1.0000
Epoch 10/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0079 - accuracy: 1.0000
Epoch 11/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0068 - accuracy: 1.0000
Epoch 12/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0060 - accuracy: 1.0000
Epoch 13/21
68/68 [==============================] - 6s 94ms/step - loss: 0.0053 - accuracy: 1.0000
Epoch 14/21
68/68 [==============================] - 6s 92ms/step - loss: 0.0047 - accuracy: 1.0000
Epoch 15/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0042 - accuracy: 1.0000
Epoch 16/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0038 - accuracy: 1.0000
Epoch 17/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 18/21
68/68 [==============================] - 6s 94ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 19/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0029 - accuracy: 1.0000
Epoch 20/21
68/68 [==============================] - 6s 93ms/step - loss: 0.0027 - accuracy: 1.0000
Epoch 21/21
68/68 [==============================] - 6s 92ms/step - loss: 0.0024 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.97%
Epoch 2: Class_accuracy: 78.71%
Epoch 3: Class_accuracy: 78.81%
Epoch 4: Class_accuracy: 76.87%
Epoch 5: Class_accuracy: 72.67%
Epoch 6: Class_accuracy: 70.01%
Epoch 7: Class_accuracy: 72.88%
Epoch 8: Class_accuracy: 76.56%
Epoch 1/22
68/68 [==============================] - 9s 94ms/step - loss: 0.6667 - accuracy: 0.7618
Epoch 2/22
68/68 [==============================] - 6s 93ms/step - loss: 0.1927 - accuracy: 0.9232
Epoch 3/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0698 - accuracy: 0.9876
Epoch 4/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0352 - accuracy: 0.9982
Epoch 5/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0217 - accuracy: 0.9995
Epoch 6/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0161 - accuracy: 1.0000
Epoch 7/22
68/68 [==============================] - 6s 92ms/step - loss: 0.0128 - accuracy: 1.0000
Epoch 8/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0105 - accuracy: 1.0000
Epoch 9/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0088 - accuracy: 1.0000
Epoch 10/22
68/68 [==============================] - 6s 94ms/step - loss: 0.0075 - accuracy: 1.0000
Epoch 11/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0065 - accuracy: 1.0000
Epoch 12/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0056 - accuracy: 1.0000
Epoch 13/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0050 - accuracy: 1.0000
Epoch 14/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0045 - accuracy: 1.0000
Epoch 15/22
68/68 [==============================] - 6s 94ms/step - loss: 0.0040 - accuracy: 1.0000
Epoch 16/22
68/68 [==============================] - 6s 92ms/step - loss: 0.0036 - accuracy: 1.0000
Epoch 17/22
68/68 [==============================] - 6s 94ms/step - loss: 0.0033 - accuracy: 1.0000
Epoch 18/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0030 - accuracy: 1.0000
Epoch 19/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0027 - accuracy: 1.0000
Epoch 20/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0025 - accuracy: 1.0000
Epoch 21/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0023 - accuracy: 1.0000
Epoch 22/22
68/68 [==============================] - 6s 93ms/step - loss: 0.0021 - accuracy: 1.0000
Epoch 1: Class_accuracy: 75.44%
Epoch 2: Class_accuracy: 76.25%
Epoch 3: Class_accuracy: 76.15%
Epoch 4: Class_accuracy: 75.95%
Epoch 5: Class_accuracy: 71.14%
Epoch 6: Class_accuracy: 67.76%
Epoch 7: Class_accuracy: 68.58%
Epoch 8: Class_accuracy: 72.36%
Epoch 1/23
68/68 [==============================] - 9s 93ms/step - loss: 0.7067 - accuracy: 0.7453
Epoch 2/23
68/68 [==============================] - 6s 93ms/step - loss: 0.2192 - accuracy: 0.9099
Epoch 3/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0685 - accuracy: 0.9871
Epoch 4/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0385 - accuracy: 0.9986
Epoch 5/23
68/68 [==============================] - 6s 92ms/step - loss: 0.0239 - accuracy: 1.0000
Epoch 6/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0178 - accuracy: 1.0000
Epoch 7/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0136 - accuracy: 1.0000
Epoch 8/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0111 - accuracy: 1.0000
Epoch 9/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0093 - accuracy: 1.0000
Epoch 10/23
68/68 [==============================] - 6s 92ms/step - loss: 0.0079 - accuracy: 1.0000
Epoch 11/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0068 - accuracy: 1.0000
Epoch 12/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0060 - accuracy: 1.0000
Epoch 13/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0053 - accuracy: 1.0000
Epoch 14/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0047 - accuracy: 1.0000
Epoch 15/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0042 - accuracy: 1.0000
Epoch 16/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0038 - accuracy: 1.0000
Epoch 17/23
68/68 [==============================] - 6s 92ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 18/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 19/23
68/68 [==============================] - 6s 92ms/step - loss: 0.0029 - accuracy: 1.0000
Epoch 20/23
68/68 [==============================] - 6s 94ms/step - loss: 0.0026 - accuracy: 1.0000
Epoch 21/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0024 - accuracy: 1.0000
Epoch 22/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0022 - accuracy: 1.0000
Epoch 23/23
68/68 [==============================] - 6s 93ms/step - loss: 0.0021 - accuracy: 1.0000
Epoch 1: Class_accuracy: 76.25%
Epoch 2: Class_accuracy: 77.69%
Epoch 3: Class_accuracy: 79.12%
Epoch 4: Class_accuracy: 78.30%
Epoch 5: Class_accuracy: 77.69%
Epoch 6: Class_accuracy: 70.42%
Epoch 7: Class_accuracy: 66.53%
Epoch 8: Class_accuracy: 69.70%
Epoch 1/24
68/68 [==============================] - 9s 96ms/step - loss: 0.7384 - accuracy: 0.7444
Epoch 2/24
68/68 [==============================] - 6s 95ms/step - loss: 0.1987 - accuracy: 0.9200
Epoch 3/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0761 - accuracy: 0.9839
Epoch 4/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0373 - accuracy: 0.9986
Epoch 5/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0241 - accuracy: 1.0000
Epoch 6/24
68/68 [==============================] - 6s 96ms/step - loss: 0.0174 - accuracy: 1.0000
Epoch 7/24
68/68 [==============================] - 6s 96ms/step - loss: 0.0139 - accuracy: 1.0000
Epoch 8/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0112 - accuracy: 1.0000
Epoch 9/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0094 - accuracy: 1.0000
Epoch 10/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0080 - accuracy: 1.0000
Epoch 11/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0068 - accuracy: 1.0000
Epoch 12/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0059 - accuracy: 1.0000
Epoch 13/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0052 - accuracy: 1.0000
Epoch 14/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0047 - accuracy: 1.0000
Epoch 15/24
68/68 [==============================] - 6s 93ms/step - loss: 0.0042 - accuracy: 1.0000
Epoch 16/24
68/68 [==============================] - 6s 93ms/step - loss: 0.0038 - accuracy: 1.0000
Epoch 17/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 18/24
68/68 [==============================] - 6s 93ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 19/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0029 - accuracy: 1.0000
Epoch 20/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0026 - accuracy: 1.0000
Epoch 21/24
68/68 [==============================] - 6s 95ms/step - loss: 0.0024 - accuracy: 1.0000
Epoch 22/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0023 - accuracy: 1.0000
Epoch 23/24
68/68 [==============================] - 6s 92ms/step - loss: 0.0021 - accuracy: 1.0000
Epoch 24/24
68/68 [==============================] - 6s 94ms/step - loss: 0.0019 - accuracy: 1.0000
Epoch 1: Class_accuracy: 74.31%
Epoch 2: Class_accuracy: 75.74%
Epoch 3: Class_accuracy: 74.00%
Epoch 4: Class_accuracy: 72.88%
Epoch 5: Class_accuracy: 72.26%
Epoch 6: Class_accuracy: 72.57%
Epoch 7: Class_accuracy: 72.47%
Epoch 8: Class_accuracy: 72.06%
Epoch 1/25
68/68 [==============================] - 9s 94ms/step - loss: 0.7089 - accuracy: 0.7605
Epoch 2/25
68/68 [==============================] - 6s 94ms/step - loss: 0.2015 - accuracy: 0.9200
Epoch 3/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0768 - accuracy: 0.9821
Epoch 4/25
68/68 [==============================] - 6s 96ms/step - loss: 0.0369 - accuracy: 0.9977
Epoch 5/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0231 - accuracy: 0.9995
Epoch 6/25
68/68 [==============================] - 6s 93ms/step - loss: 0.0169 - accuracy: 1.0000
Epoch 7/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0132 - accuracy: 1.0000
Epoch 8/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0108 - accuracy: 1.0000
Epoch 9/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0090 - accuracy: 1.0000
Epoch 10/25
68/68 [==============================] - 6s 95ms/step - loss: 0.0078 - accuracy: 1.0000
Epoch 11/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0067 - accuracy: 1.0000
Epoch 12/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0059 - accuracy: 1.0000
Epoch 13/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0052 - accuracy: 1.0000
Epoch 14/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0046 - accuracy: 1.0000
Epoch 15/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0042 - accuracy: 1.0000
Epoch 16/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0037 - accuracy: 1.0000
Epoch 17/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0034 - accuracy: 1.0000
Epoch 18/25
68/68 [==============================] - 6s 95ms/step - loss: 0.0031 - accuracy: 1.0000
Epoch 19/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0028 - accuracy: 1.0000
Epoch 20/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0026 - accuracy: 1.0000
Epoch 21/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0024 - accuracy: 1.0000
Epoch 22/25
68/68 [==============================] - 6s 95ms/step - loss: 0.0022 - accuracy: 1.0000
Epoch 23/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0021 - accuracy: 1.0000
Epoch 24/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0019 - accuracy: 1.0000
Epoch 25/25
68/68 [==============================] - 6s 94ms/step - loss: 0.0018 - accuracy: 1.0000
Epoch 1: Class_accuracy: 74.51%
Epoch 2: Class_accuracy: 76.05%
Epoch 3: Class_accuracy: 76.77%
Epoch 4: Class_accuracy: 77.58%
Epoch 5: Class_accuracy: 76.66%
Epoch 6: Class_accuracy: 76.36%
Epoch 7: Class_accuracy: 76.36%
Epoch 8: Class_accuracy: 76.05%
In [40]:
np_hmt_transfer_teacher_model_acc = np.array(hmt_transfer_teacher_model_acc)
print(np_hmt_transfer_teacher_model_acc)
np_hmt_transfer_student_model_acc = np.array(hmt_transfer_student_model_acc)
print(np_hmt_transfer_student_model_acc)

plt.plot(epoch_array, hmt_transfer_teacher_model_acc, label = 'teacher')
plt.plot(epoch_array, hmt_transfer_student_model_acc, label = 'student')
plt.legend()
plt.title("Accuracy vs teacher train epoch")
plt.xlabel('Epoch number')
plt.ylabel('Accuracy')
plt.show()
[76.66325  77.175026 79.32446  79.529175 79.32446  79.63152  80.85977
 78.60798  81.16684  80.24565  80.24565  79.73388  79.42682  81.98567
 81.4739   79.529175 79.529175 81.371544 81.78097  80.450356 80.65507
 80.85977  80.65507  80.348    79.63152 ]
[68.372574 73.18321  77.58444  67.04196  67.14432  77.07268  68.27021
 71.95496  66.325485 70.419655 66.018425 68.47492  67.96315  71.95496
 68.986694 71.34084  73.387924 69.1914   67.553734 70.21494  76.560905
 72.36438  69.70317  72.05732  76.04913 ]